{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import subprocess\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from PIL import Image\n",
    "from sklearn.model_selection import StratifiedKFold, StratifiedShuffleSplit\n",
    "import json\n",
    "import exifread\n",
    "import subprocess\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "from PIL import Image\n",
    "\n",
    "import env\n",
    "from utils import KaggleCameraDataset, progress_iter, RNG\n",
    "\n",
    "%matplotlib inline\n",
    "%load_ext autoreload\n",
    "%autoreload 2"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### `N_IMAGES_NEEDED`"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[3841, 4253, 4026, 4027, 4206, 3921, 3291, 4070, 3931, 3734]"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "MAX_IMAGES = 5000\n",
    "N_IMAGES = [991, 651, 767, 773, 595, 873, 1490, 751, 888, 1068]\n",
    "N_IMAGES = [x+y for x, y in zip(N_IMAGES, [198-30, 126-30, 237-30, 232-32, 229-30, 236-30, 249-30, 209-30, 214-33, 228-30])]\n",
    "N_IMAGES_NEEDED = [5000-x for x in N_IMAGES]\n",
    "N_IMAGES_NEEDED"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## some routines"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def jpg_quality(filepath):\n",
    "    bash_cmd = \"identify -verbose {0} | grep Quality\".format(filepath)\n",
    "    process = subprocess.Popen(bash_cmd.split(), stdout=subprocess.PIPE)\n",
    "    output, error = process.communicate()\n",
    "    output = filter(lambda s: 'Quality' in s, output.split('\\n'))\n",
    "    if not output: \n",
    "        return None\n",
    "    return int(filter(lambda c: c.isdigit(), output[0]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[(0, 'HTC-1-M7'),\n",
       " (1, 'LG-Nexus-5x'),\n",
       " (2, 'Motorola-Droid-Maxx'),\n",
       " (3, 'Motorola-Nexus-6'),\n",
       " (4, 'Motorola-X'),\n",
       " (5, 'Samsung-Galaxy-Note3'),\n",
       " (6, 'Samsung-Galaxy-S4'),\n",
       " (7, 'Sony-NEX-7'),\n",
       " (8, 'iPhone-4s'),\n",
       " (9, 'iPhone-6')]"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "list(enumerate(KaggleCameraDataset.target_labels()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def resolutions(): # with all possible rotations\n",
    "    return [\n",
    "        [(1520, 2688), (2688, 1520)], # 0\n",
    "        [(3024, 4032), (4032, 3024)], # 1\n",
    "        [(2432, 4320), (4320, 2432)], # 2\n",
    "        [(1040, 780), (3088, 4130), (4130, 3088), (3120, 4160), (4160, 3120)], # 3\n",
    "        [(3120, 4160), (4160, 3120), (4160, 2340)], # 4\n",
    "        [(4128, 2322)], # 5\n",
    "        [(4128, 2322)], # 6\n",
    "        [(6000, 4000)], # 7\n",
    "        [(3264, 2448)], # 8\n",
    "        [(3264, 2448)], # 9\n",
    "    ]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### load data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "andreas_data = KaggleCameraDataset('../data/andreas/', train=True)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def make_stats(dataset, c=None, partial_stats=None):\n",
    "    x_fpaths = dataset.X\n",
    "    stats = partial_stats or {}\n",
    "    for i in progress_iter(range(len(dataset)), verbose=True):\n",
    "        try:\n",
    "            if c is not None and dataset.y[i] != c:\n",
    "                continue\n",
    "            if partial_stats is not None and x_fpaths[i] in map(lambda d: d['x_fpath'], stats.values()):\n",
    "                continue\n",
    "            x, y = dataset[i]\n",
    "            x = np.asarray(x, dtype=np.uint8)\n",
    "            if len(x.shape) != 3 or x.shape[0] < 1024 or x.shape[1] < 1024:\n",
    "                continue\n",
    "            d = {}\n",
    "            d['x_fpath'] = x_fpaths[i]\n",
    "            d['y'] = y\n",
    "            d['res'] = (x.shape[0], x.shape[1])\n",
    "            d['res_valid'] = d['res'] in resolutions()[y]\n",
    "            d['quality'] = 95 # jpg_quality(x_fpaths[i])\n",
    "            stats[i] = d\n",
    "        except:\n",
    "            pass\n",
    "    return stats"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 1 Nexus 5 stats"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "andreas_stats_1 = make_stats(andreas_data, c=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print len(andreas_stats_1)\n",
    "print min(map(lambda d: d['quality'], andreas_stats_1.values()))\n",
    "print sum(map(lambda d: d['res_valid'], andreas_stats_1.values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# with open('../data/andreas_stats_1.json', 'w') as f:\n",
    "#     json.dump(andreas_stats_1, f, indent=4, sort_keys=True)\n",
    "with open('../data/andreas_stats_1.json') as f:\n",
    "    andreas_stats_1 = json.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 0 (HTC)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "andreas_stats_0 = make_stats(andreas_data, 0, andreas_stats_0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print len(andreas_stats_0)\n",
    "print min(map(lambda d: d['quality'], andreas_stats_0.values()))\n",
    "print sum(map(lambda d: d['res_valid'], andreas_stats_0.values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# with open('../data/andreas_stats_0.json', 'w') as f:\n",
    "#     json.dump(andreas_stats_0, f, indent=4, sort_keys=True)\n",
    "with open('../data/andreas_stats_0.json') as f:\n",
    "    andreas_stats_0 = json.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 9 (iPhone 6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "andreas_stats_9 = make_stats(andreas_data, 9, andreas_stats_9)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print len(andreas_stats_9)\n",
    "print min(map(lambda d: d['quality'], andreas_stats_9.values()))\n",
    "print sum(map(lambda d: d['res_valid'], andreas_stats_9.values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# with open('../data/andreas_stats_9.json', 'w') as f:\n",
    "#     json.dump(andreas_stats_9, f, indent=4, sort_keys=True)\n",
    "with open('../data/andreas_stats_9.json') as f:\n",
    "    andreas_stats_9 = json.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 2 (Droid Maxx)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 126,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "0a1381dfa514435d964933cbf797d82d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "andreas_stats_2 = make_stats(andreas_data, 2)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 128,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5660\n",
      "85\n",
      "5660\n"
     ]
    }
   ],
   "source": [
    "print len(andreas_stats_2)\n",
    "print min(map(lambda d: d['quality'], andreas_stats_2.values()))\n",
    "print sum(map(lambda d: d['res_valid'], andreas_stats_2.values()))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# with open('../data/andreas_stats_2.json', 'w') as f:\n",
    "#     json.dump(andreas_stats_2, f, indent=4, sort_keys=True)\n",
    "with open('../data/andreas_stats_2.json') as f:\n",
    "    andreas_stats_2 = json.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 8 (iPhone 4S)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 133,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "f516ee697a6843a6ab64c71dc4eb9d97",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "9098/|/ 58%|| 9098/15691 [00:20<00:14, 454.10it/s]\n"
     ]
    }
   ],
   "source": [
    "andreas_stats_8 = make_stats(andreas_data, 8)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "6575\n",
      "None\n",
      "1803\n"
     ]
    }
   ],
   "source": [
    "print len(andreas_stats_8)\n",
    "print min(map(lambda d: d['quality'], andreas_stats_8.values()))\n",
    "print sum(map(lambda d: d['res_valid'], andreas_stats_8.values()))\n",
    "\n",
    "# with open('../data/andreas_stats_8.json', 'w') as f:\n",
    "#     json.dump(andreas_stats_8, f, indent=4, sort_keys=True)\n",
    "with open('../data/andreas_stats_8.json') as f:\n",
    "    andreas_stats_8 = json.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 6 (S4)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 136,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "623c57ffd74b4f2ba09cd7fced574b76",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5663/|/ 22%|| 5663/25514 [00:20<01:10, 281.75it/s]\n"
     ]
    }
   ],
   "source": [
    "andreas_stats_6 = make_stats(andreas_data, 6)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# print len(andreas_stats_6)\n",
    "# print min(map(lambda d: d['quality'], andreas_stats_6.values()))\n",
    "# print sum(map(lambda d: d['res_valid'], andreas_stats_6.values()))\n",
    "\n",
    "# with open('../data/andreas_stats_6.json', 'w') as f:\n",
    "#     json.dump(andreas_stats_6, f, indent=4, sort_keys=True)\n",
    "with open('../data/andreas_stats_6.json') as f:\n",
    "    andreas_stats_6 = json.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# andreas_stats_3 = make_stats(andreas_data, 3)\n",
    "\n",
    "# print len(andreas_stats_3)\n",
    "# print min(map(lambda d: d['quality'], andreas_stats_3.values()))\n",
    "# print sum(map(lambda d: d['res_valid'], andreas_stats_3.values()))\n",
    "\n",
    "# with open('../data/andreas_stats_3.json', 'w') as f:\n",
    "#     json.dump(andreas_stats_3, f, indent=4, sort_keys=True)\n",
    "with open('../data/andreas_stats_3.json') as f:\n",
    "    andreas_stats_3 = json.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# andreas_stats_5 = make_stats(andreas_data, 5)\n",
    "\n",
    "# print len(andreas_stats_5)\n",
    "# print min(map(lambda d: d['quality'], andreas_stats_5.values()))\n",
    "# print sum(map(lambda d: d['res_valid'], andreas_stats_5.values()))\n",
    "\n",
    "# with open('../data/andreas_stats_5.json', 'w') as f:\n",
    "#     json.dump(andreas_stats_5, f, indent=4, sort_keys=True)\n",
    "with open('../data/andreas_stats_5.json') as f:\n",
    "    andreas_stats_5 = json.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d072cf3dea3c4ceb9b0fe1086c0aa833",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "2076\n",
      "95\n",
      "205\n"
     ]
    }
   ],
   "source": [
    "andreas_stats_7 = make_stats(andreas_data, 7)\n",
    "\n",
    "print len(andreas_stats_7)\n",
    "print min(map(lambda d: d['quality'], andreas_stats_7.values()))\n",
    "print sum(map(lambda d: d['res_valid'], andreas_stats_7.values()))\n",
    "\n",
    "with open('../data/andreas_stats_7.json', 'w') as f:\n",
    "    json.dump(andreas_stats_7, f, indent=4, sort_keys=True)\n",
    "with open('../data/andreas_stats_7.json') as f:\n",
    "    andreas_stats_7 = json.load(f)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# filter according to quality"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "QUALITY_THRESHOLD = 93 # 95 !!!"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_1 = []\n",
    "for i, d in sorted(andreas_stats_1.items()):\n",
    "    if d['quality'] and d['quality'] >= QUALITY_THRESHOLD:\n",
    "        train_1.append((d['x_fpath'], d['y'], d['quality'], d['res'], d['res_valid']))\n",
    "print len(train_1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_0 = []\n",
    "for i, d in sorted(andreas_stats_0.items()):\n",
    "    if d['quality'] and d['quality'] >= 95: # QUALITY_THRESHOLD:\n",
    "        train_0.append((d['x_fpath'], d['y'], d['quality'], d['res'], d['res_valid']))\n",
    "print len(train_0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train_9 = []\n",
    "for i, d in sorted(andreas_stats_9.items()):\n",
    "    if d['quality'] and d['quality'] >= QUALITY_THRESHOLD:\n",
    "        train_9.append((d['x_fpath'], d['y'], d['quality'], d['res'], d['res_valid']))\n",
    "print len(train_9)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5355\n"
     ]
    }
   ],
   "source": [
    "train_2 = []\n",
    "for i, d in sorted(andreas_stats_2.items()):\n",
    "    if d['quality'] and d['quality'] >= 95: # QUALITY_THRESHOLD:\n",
    "        train_2.append((d['x_fpath'], d['y'], d['quality'], d['res'], d['res_valid']))\n",
    "print len(train_2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3149\n"
     ]
    }
   ],
   "source": [
    "train_6 = []\n",
    "for i, d in sorted(andreas_stats_6.items()):\n",
    "    if d['quality'] and d['quality'] >= 95: # QUALITY_THRESHOLD:\n",
    "        train_6.append((d['x_fpath'], d['y'], d['quality'], d['res'], d['res_valid']))\n",
    "print len(train_6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5809\n"
     ]
    }
   ],
   "source": [
    "train_8 = []\n",
    "for i, d in sorted(andreas_stats_8.items()):\n",
    "    if d['quality'] and d['quality'] >= 96: # QUALITY_THRESHOLD:\n",
    "        train_8.append((d['x_fpath'], d['y'], d['quality'], d['res'], d['res_valid']))\n",
    "print len(train_8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3060\n"
     ]
    }
   ],
   "source": [
    "train_3 = []\n",
    "for i, d in sorted(andreas_stats_3.items()):\n",
    "    if d['quality'] and d['quality'] >= 95: # QUALITY_THRESHOLD:\n",
    "        train_3.append((d['x_fpath'], d['y'], d['quality'], d['res'], d['res_valid']))\n",
    "print len(train_3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3653\n"
     ]
    }
   ],
   "source": [
    "train_5 = []\n",
    "for i, d in sorted(andreas_stats_5.items()):\n",
    "    if d['quality'] and d['quality'] >= 95: # QUALITY_THRESHOLD:\n",
    "        train_5.append((d['x_fpath'], d['y'], d['quality'], d['res'], d['res_valid']))\n",
    "print len(train_5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2076\n"
     ]
    }
   ],
   "source": [
    "train_7 = []\n",
    "for i, d in sorted(andreas_stats_7.items()):\n",
    "    if d['quality'] and d['quality'] >= 95: # QUALITY_THRESHOLD:\n",
    "        train_7.append((d['x_fpath'], d['y'], d['quality'], d['res'], d['res_valid']))\n",
    "print len(train_7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# filter according to resolutions"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def distance(res, c):\n",
    "    return min(min([np.sqrt((res[0]-r[0])**2+(res[1]-r[1])**2) for r in resolutions()[c]]),\n",
    "               min([np.sqrt((res[0]-r[1])**2+(res[1]-r[0])**2) for r in resolutions()[c]]))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "filter(lambda (x,y,q,r,rv): not rv, train_1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "filter(lambda (x,y,q,r,rv): not rv, train_0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "incorrect = filter(lambda (x,y,q,r,rv): not rv, train_9)\n",
    "print len(incorrect)\n",
    "dists = map(lambda (x,y,q,r,rv): distance(r, 9), incorrect)\n",
    "dists.sort()\n",
    "plt.hist(dists, bins=24);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "dists[:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print len(train_9)\n",
    "train_9 = filter(lambda (x,y,q,r,rv): rv or (not rv and (np.abs(distance(r, 9)) < 0.01)), train_9)\n",
    "print len(train_9)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "[]"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "filter(lambda (x,y,q,r,rv): not rv, train_2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3108\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYMAAAEACAYAAABRQBpkAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEvJJREFUeJzt3X+s3XV9x/Hnq7Q40VlQR7u1BZwywGVZJROYbtl1ghaW\nybIEBbfww5mQCWps4gCXrO1cAizTOeOYmWNSndohm8IyB4XUY7Zl/BqtdLZA/QG0dL3ipCy4xAm8\n98f5XDy7nHLPvff03FN5PpKT+/2+z+f7/b7PPefe1/l+v+d7b6oKSdLz26KFbkCStPAMA0mSYSBJ\nMgwkSRgGkiQMA0kSA4RBkhckuSPJ1iTbk6xr9eOS3J7k/iSfS7K41Q9PsinJriT/luSYnnVd0eo7\nk7zp4D0sSdJszBgGVfV94A1V9RpgNXBmklOBq4EPVdUJwH7gd9oivwN8t6qOBz4C/DFAklcDbwVO\nAs4ErkmSIT8eSdIcDHSYqKr+p02+AFgMFPAG4O9afSPwG2367DYPcAPwq236LcCmqnqyqh4EdgGn\nzKd5SdJwDBQGSRYl2QrsA24FvgHsr6qn25A9wIo2vQLYDVBVTwGPJ3lpb715pGcZSdICGnTP4Ol2\nmGgl3XfzJ/Ub1r72O/RTz1GXJC2wxbMZXFX/neQrwGnAkUkWtb2DlcDeNmwPsArYm+QwYGlVPZZk\nqj6ld5lnJDEgJGkOqmrO52EH+TTRy5MsbdMvBE4HdgBfBs5pwy4AbmzTN7V52v1beurntk8bvQJ4\nFXBnv21W1djf1q1bt+A92Kd92qc9Tt3ma5A9g58ENiZZRDc8/raqvpRkJ7ApyQeBrcC1bfy1wKeT\n7AL+Czi3/YLfkeR6ukHyA+BdNYxHIEmatxnDoKq2Ayf3qX8LOLVP/ft0P0Lab11XAlfOvk1J0sHk\nFchzNDExsdAtDMQ+h8s+h+tQ6PNQ6HEYMm5HapJ49EiSZikJdTBPIEuSfvQZBpKk2V1nMCrvf/8V\nsxqfwLvf/S5WrVo182BJ0rOMZRj8yZ+8ZFbjDzvsH1m+/CdYu3btQepIkn60jWUYwOz2DBYtevQg\n9SFJzw+eM5AkGQaSJMNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEY\nSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCQxQBgkWZlkS5IdSbYneXerr0uyJ8k97bamZ5kr\nkuxKsjPJm3rqa5Lcl+SBJJcdnIckSZqtxQOMeRJYW1XbkrwY+Pckt7b7PlxVH+4dnOQk4K3AScBK\n4LYkxwMBPga8EdgL3JXkxqq6b0iPRZI0RzOGQVXtA/a16SeS7ARWtLvTZ5GzgU1V9STwYJJdwClt\n7K6qegggyaY21jCQpAU2q3MGSY4DVgN3tNIlSbYl+askS1ttBbC7Z7FHWm16fQ8/DBVJ0gIaOAza\nIaIbgPdW1RPANcArq2o13T2HD00N7bN4PUddkrTABjlnQJLFdIPg01V1I0BVPdoz5BPAP7TpPcCq\nnvtW0j1HEOCYPvU+1vdMT7SbJGlKp9Oh0+kMbX2pmvnNeZJPAd+pqrU9teXtfAJJ3ge8tqrenuTV\nwGeAU+keBroVOJ7uXsj9dE8g/ydwJ3BeVe2ctq2a7Q7DkiVrueqqlaxdu3bmwZL0IygJVdXvCMxA\nZtwzSPJ64LeA7Um20v1N/QHg7UlWA08DDwIXA1TVjiTXAzuAHwDvqm7iPJXkUmAz3WC4dnoQSJIW\nxiCfJvpX4LA+d938HMtcCVzZp34zcMJsGpQkHXxegSxJMgwkSYaBJAnDQJKEYSBJwjCQJGEYSJIw\nDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAk\nYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJYoAwSLIyyZYkO5JsT/KeVj8qyeYk9ye5JcnSnmU+\nmmRXkm1JVvfUL0jyQFvm/IPzkCRJszXInsGTwNqqejXwi8AlSU4ELgduq6oTgC3AFQBJzgReWVXH\nAxcDH2/1o4A/AF4LnAqs6w0QSdLCmTEMqmpfVW1r008AO4GVwNnAxjZsY5unff1UG38HsDTJMuDN\nwOaqeryq9gObgTVDfCySpDma1TmDJMcBq4HbgWVVNQndwACObsNWALt7FtvTatPrj7SaJGmBLR50\nYJIXAzcA762qJ5LUgYb2ma8+dVq9j/U90xPtJkma0ul06HQ6Q1vfQGGQZDHdIPh0Vd3YypNJllXV\nZJLlwLdbfQ+wqmfxlcDeVp+YVv9y/y2uH6x7SXqempiYYGJi4pn5DRs2zGt9gx4m+mtgR1X9WU/t\nJuDCNn0hcGNP/XyAJKcB+9vhpFuAM5IsbSeTz2g1SdICm3HPIMnrgd8CtifZSvfQzgeAq4Hrk7wD\neBg4B6CqvpTkrCRfB74HXNTqjyX5IHB3W8eGdiJZkrTAZgyDqvpX4LAD3H36AZa59AD164DrBuxN\nkjQiXoEsSTIMJEmGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAk\nYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaSJAwDSRKGgSQJw0CSxABh\nkOTaJJNJ7u2prUuyJ8k97bam574rkuxKsjPJm3rqa5Lcl+SBJJcN/6FIkuZqkD2DTwJv7lP/cFWd\n3G43AyQ5CXgrcBJwJnBNuhYBH2vr+VngvCQnDuURSJLmbfFMA6rqX5Ic2+eu9KmdDWyqqieBB5Ps\nAk5pY3dV1UMASTa1sffNuXNJ0tDM55zBJUm2JfmrJEtbbQWwu2fMI602vb6n1SRJY2DGPYMDuAb4\nw6qqJH8EfAh4J/33For+oVMHXv36numJdpMkTel0OnQ6naGtb05hUFWP9sx+AviHNr0HWNVz30pg\nL92QOKZP/QDWz6UtSXremJiYYGJi4pn5DRs2zGt9gx4mCj3v+pMs77nvN4H/aNM3AecmOTzJK4BX\nAXcCdwGvSnJsksOBc9tYSdIYmHHPIMln6R6neVmSh4F1wBuSrAaeBh4ELgaoqh1Jrgd2AD8A3lVV\nBTyV5FJgM90Auraqdg7/4UiS5mKQTxO9vU/5k88x/krgyj71m4ETZtWdJGkkvAJZkmQYSJIMA0kS\nhoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJwjCQJGEYSJIwDCRJGAaS\nJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkiQHCIMm1SSaT3NtTOyrJ5iT3\nJ7klydKe+z6aZFeSbUlW99QvSPJAW+b84T8USdJcDbJn8EngzdNqlwO3VdUJwBbgCoAkZwKvrKrj\ngYuBj7f6UcAfAK8FTgXW9QaIJGlhzRgGVfUvwGPTymcDG9v0xjY/Vf9UW+4OYGmSZXTDZHNVPV5V\n+4HNwJr5ty9JGoa5njM4uqomAapqH3B0q68AdveM29Nq0+uPtJokaQwsHvL60me++tRp9QNY3zM9\n0W6SpCmdTodOpzO09c01DCaTLKuqySTLgW+3+h5gVc+4lcDeVp+YVv/ygVe/fo5tSdLzw8TEBBMT\nE8/Mb9iwYV7rG/QwUfj/7+5vAi5s0xcCN/bUzwdIchqwvx1OugU4I8nSdjL5jFaTJI2BGfcMknyW\n7rv6lyV5GFgHXAV8Psk7gIeBcwCq6ktJzkrydeB7wEWt/liSDwJ30z08tKGdSJYkjYEZw6Cq3n6A\nu04/wPhLD1C/Drhu0MYkSaPjFciSJMNAkmQYSJIwDCRJGAaSJAwDSRKGgSQJw0CShGEgScIwkCRh\nGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJAkYRhIkjAMJEkYBpIkDANJEoaBJAnDQJKEYSBJ\nwjCQJGEYSJIwDCRJzDMMkjyY5KtJtia5s9WOSrI5yf1JbkmytGf8R5PsSrItyer5Ni9JGo757hk8\nDUxU1Wuq6pRWuxy4rapOALYAVwAkORN4ZVUdD1wMfHye25YkDcl8wyB91nE2sLFNb2zzU/VPAVTV\nHcDSJMvmuX1J0hDMNwwKuCXJXUne2WrLqmoSoKr2AUe3+gpgd8+yj7SaJGmBLZ7n8q+rqn1JfgLY\nnOR+ugHRT/rUDjRWkjRC8wqD9s6fqno0yReBU4DJJMuqajLJcuDbbfgeYFXP4iuBvf3XvL5neqLd\nJElTOp0OnU5naOtL1dzenCc5AlhUVU8keRGwGdgAvBH4blVdneRy4MiqujzJWcAlVfVrSU4DPlJV\np/VZb812h2HJkrVcddVK1q5dO6fHIkmHuiRUVb8jMAOZz57BMuAL3V/eLAY+U1Wbk9wNXJ/kHcDD\nwDkAVfWlJGcl+TrwPeCieWxbkjREcw6DqvoW8KxrBarqu8DpB1jm0rluT5J08HgFsiTJMJAkGQaS\nJAwDSRKGgSQJw0CShGEgScIwkCRhGEiSMAwkSRgGkiQMA0kShoEkCcNAkoRhIEnCMJCkkVi+/DiS\nzOq2fPlxI+tvXv8DWZI0mMnJh5jtv/SdnJzzf7GcNfcMJEmGgSTJMJAkYRhIkjAMJEkYBpIkDANJ\nEoaBJAnDQJKEYSBJwjCQJLEAYZBkTZL7kjyQ5LJRb1+S9GwjDYMki4CPAW8GfhY4L8mJo+xhWDqd\nzkK3MBD7HC77HK5Doc9DocdhGPWewSnArqp6qKp+AGwCzh5xD0NxqLxA7HO47HO4DoU+D4Ueh2HU\nYbAC2N0zv6fVJEkLaNT/z6DfH+d+1h/4fslLfn1WK/3f//0aS5a8b649SdLzXqpm988W5rWx5DRg\nfVWtafOXA1VVV/eMGV1DkvQjpKrm/N9wRh0GhwH3A28E/hO4EzivqnaOrAlJ0rOM9DBRVT2V5FJg\nM93zFdcaBJK08Ea6ZyBJGk9jdQXyOF2QluTaJJNJ7u2pHZVkc5L7k9ySZGnPfR9NsivJtiSrR9Tj\nyiRbkuxIsj3Je8a0zxckuSPJ1tbnulY/Lsntrc/PJVnc6ocn2dT6/Lckx4yiz55+FyW5J8lN49pn\nkgeTfLV9T+9stbF63tt2lyb5fJKdSb6W5NRx6zPJz7Tv4z3t6+NJ3jNufbbtvi/JfyS5N8ln2mtw\nOK/PqhqLG91g+jpwLLAE2AacuID9/BKwGri3p3Y18Htt+jLgqjZ9JvCPbfpU4PYR9bgcWN2mX0z3\nfMyJ49Zn294R7ethwO1t+38LnNPqfwFc3KZ/F7imTb8N2DTi5/59wN8AN7X5sesT+CZw1LTaOD7v\n1wEXtenFwNJx7LOn30XAXmDVuPUJ/FR73g/veV1eMKzX50i/0TM80NOAf+qZvxy4bIF7Opb/Hwb3\nAcva9HJgZ5v+OPC2nnE7p8aNuN8vAqePc5/AEcDddC9A/DawaPrzD9wMnNqmDwMeHWF/K4FbgQl+\nGAaPjmGf3wJeNq02Vs878OPAN/rUx6rPab29CfjnceyTbhg8BBxFN1hvAs4Y1s/ROB0mOhQuSDu6\nqiYBqmofcHSrT+/9EUbce5Lj6O7J3E73hTlWfbZDL1uBfXR/2X4D2F9VT7chvc/3M31W1VPA/iQv\nHUWfwJ8C76dd/5LkZcBjY9hnAbckuSvJO1tt3J73nwa+k+ST7RDMXyY5Ygz77PU24LNteqz6rKq9\nwIeAh9s2HwfuYUg/R+MUBgNdkDamFrT3JC8GbgDeW1VPPMe2F6zPqnq6ql5D9533KcBJz9HL9D7D\nCPpM8mvAZFVt6+khffpZ0D6b11XVLwBnAZck+eXn2PZCPe+LgZOBP6+qk4Hv0d3jH7c+uxtPlgBv\nAT4/w7YXpM8kR9L98z3H0t1LeBHdQ1YH6mVWr89xCoM9QO8JjpV0j92Nk8kkywCSLKe7ewbd3lf1\njBtZ7+1k0Q3Ap6vqxnHtc0pV/TfwFbq7s0em+8cLp/fyTJ/pXpvykqp6bATtvR54S5JvAp8DfhX4\nCLB0zPqceqdKVT1K9/DgKYzf874H2F1Vd7f5v6MbDuPW55QzgX+vqu+0+XHr83Tgm1X13fZO/wvA\n6xjSz9E4hcFdwKuSHJvkcOBcusfEFtL0d4U3ARe26QuBG3vq58MzV1nvn9q9HIG/BnZU1Z/11Maq\nzyQvn/okRpIX0n1R7wC+DJzThl0wrc8L2vQ5wJaD3SNAVX2gqo6pqp+m+/rbUlW/PW59Jjmi7Q2S\n5EV0j3NvZ8ye97aN3Ul+ppXeCHxt3PrscR7dNwFTxq3Ph4HTkvxYkvDD7+dwXp+jPDkzwAmSNXQ/\nEbMLuHyBe/ks3YT9fnsSLqJ74ua21uOtwJE94z9G99NQXwVOHlGPrweeovvJq610jx+uAV46Zn3+\nXOttG3Av8Put/grgDuABup+IWNLqLwCub6+D24HjFuD5/xV+eAJ5rPps/Uw959unflbG7Xlv2/15\num/0tgF/T/fTROPY5wvpflDgx3tq49jnOronrO8FNtL95OVQXp9edCZJGqvDRJKkBWIYSJIMA0mS\nYSBJwjCQJGEYSJIwDCRJGAaSJOD/ANUJtCewCJQbAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fefb71a0950>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "incorrect = filter(lambda (x,y,q,r,rv): not rv, train_6)\n",
    "print len(incorrect)\n",
    "dists = map(lambda (x,y,q,r,rv): distance(r, 6), incorrect)\n",
    "dists.sort()\n",
    "plt.hist(dists, bins=24);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3149\n",
      "2915\n"
     ]
    }
   ],
   "source": [
    "print len(train_6)\n",
    "train_6 = filter(lambda (x,y,q,r,rv): rv or (not rv and (np.abs(distance(r, 6)) < 0.01)), train_6)\n",
    "print len(train_6)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5809\n",
      "5809\n"
     ]
    }
   ],
   "source": [
    "print len(train_8)\n",
    "train_8 = filter(lambda (x,y,q,r,rv): rv or (not rv and (np.abs(distance(r, 8)) < 0.01)), train_8)\n",
    "print len(train_8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3060\n",
      "1893\n"
     ]
    }
   ],
   "source": [
    "print len(train_3)\n",
    "train_3 = filter(lambda (x,y,q,r,rv): rv or (not rv and (np.abs(distance(r, 3)) < 0.01)), train_3)\n",
    "print len(train_3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3652\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYcAAAEACAYAAABYq7oeAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAEW5JREFUeJzt3X+s3XV9x/HnCyszyizVjN5IgSoyisbZsQCdZOE6FAtm\nI1nCBF34oSYkghJJFopmFmKiwwQnxCmZIkEjIjIdJWFQCdw/XAqIUEF7KWWOHwV7ZQPM4I9Fy3t/\nnG/p4X5uub2/D73PR3KS73mfz/d8P99Pvzmv8/l+z7c3VYUkSf32W+gOSJIGj+EgSWoYDpKkhuEg\nSWoYDpKkhuEgSWpMGg5JViS5I8mWJA8m+WRXX59ke5L7usfavnUuTrItyWiSk/rqa5M8lOThJBfN\nzS5JkmYqk93nkGQIGKqqzUkOAH4GnAp8CPjfqvryuPZHAdcBxwArgNuBI4AADwMnAk8BPwVOr6qH\nZnWPJEkztmSyBlW1A9jRLT+fZBQ4uHs5E6xyKnB9Vf0eeDTJNuDYru22qnoMIMn1XVvDQZIGzJSu\nOSRZCawG7u5K5yXZnOSbSZZ2tYOBJ/pWe7Krja9vZ3fISJIGyF6HQ3dK6Ubggqp6HvgacHhVraY3\ns7h8V9MJVq9XqEuSBsykp5UAkiyhFwzfqaqbAKrq6b4m3wBu7pa3A4f0vbaC3jWGAIdOUB+/LQND\nkqahqib6Ej4teztz+Bawpaqu2FXoLlTv8jfAL7rlDcDpSfZP8lbg7cA99C5Avz3JYUn2B07v2jaq\nykcV69evX/A+DMrDsXAsHItXfsy2SWcOSY4HPgI8mOR+eqeCPgN8OMlq4EXgUeDc7oN9S5IbgC3A\n74BPVK/nO5OcD2ykF0pXV9XorO+RJGnG9ubXSv8BvGaCl259hXW+CHxxgvqtwJFT6aAkaf55h/QA\nGx4eXuguDAzHYjfHYjfHYu5MehPcfEtSg9YnSRp0SagFuCAtSVpEDAdJUsNwkCQ1DAdJUsNwkCQ1\nDAdJUsNwkCQ1DAdJUsNwkCQ1DAdJUsNwkCQ1DAdJUsNwkCQ1DAdJUsNwkCQ1DAdJUsNwkCQ1DAdJ\nUsNwkCQ1DAdJUsNwkCQ1DAdJUsNwkCQ1DAdJUsNwkCQ1DAdJUsNwkCQ1DAdJUsNwkCQ1DAdJUsNw\nkCQ1DAdJUsNwkCQ1DAdJUmPScEiyIskdSbYkeTDJp7r6siQbk2xNcluSpX3rXJlkW5LNSVb31c9K\n8nC3zplzs0uSpJlKVb1yg2QIGKqqzUkOAH4GnAqcA/xPVX0pyUXAsqpal+Rk4Pyq+mCS44ArqmpN\nkmXAvcDRQLr3ObqqfjtuezVZnyRJL5eEqspsvd+SyRpU1Q5gR7f8fJJRYAW9gDiha3YtcCewrqt/\nu2t/d5KlSZYD7wU27gqDJBuBtcD3x2/zs5/9hynvyAkn/AUnnXTSlNeTJLUmDYd+SVYCq4G7gOVV\nNQa9AElyUNfsYOCJvtW2d7Xx9Se7WuMLX9h/Kt0CHueHP1zP6KjhIEmzYa/DoTuldCNwQTeD2NO5\nn/HTmgA1QZ2uPoGdfcvD3eOVbAIunKSNJO07RkZGGBkZmbP336twSLKEXjB8p6pu6spjSZZX1Vh3\nXeI3XX07cEjf6iuAp7r68Lj6nRNv8ZK9670kLVLDw8MMDw+/9PzSSy+d1fff25+yfgvYUlVX9NU2\nAGd3y2cDN/XVzwRIsgZ4rjv9dBvw/u4axDLg/V1NkjRgJp05JDke+AjwYJL76Z0K+gxwGXBDko8C\njwOnAVTVLUlOSfII8AK9XzVRVc8m+Ty9XywVcGlVPTcH+yRJmqFJf8o633rXMqbap02sWnUho6Ob\n5qRPkjToZvunrN4hLUlqGA6SpIbhIElqGA6SpIbhIElqGA6SpIbhIElqGA6SpIbhIElqGA6SpIbh\nIElqGA6SpIbhIElqGA6SpIbhIElqGA6SpIbhIElqGA6SpIbhIElqGA6SpIbhIElqGA6SpIbhIElq\nGA6SpIbhIElqGA6SpIbhIElqGA6SpIbhIElqGA6SpIbhIElqGA6SpIbhIElqGA6SpIbhIElqTBoO\nSa5OMpbkgb7a+iTbk9zXPdb2vXZxkm1JRpOc1Fdfm+ShJA8nuWj2d0WSNFv2ZuZwDfCBCepfrqqj\nu8etAEmOAv4WOAo4GfhaevYDvtq9zzuBM5KsmpU9kCTNuiWTNaiqnyQ5bIKXMkHtVOD6qvo98GiS\nbcCxXdttVfUYQJLru7YPTbvnkqQ5M5NrDucl2Zzkm0mWdrWDgSf62jzZ1cbXt3c1SdIAmm44fA04\nvKpWAzuAy7v6RLOJeoW6JGkATXpaaSJV9XTf028AN3fL24FD+l5bATxFLxwOnaC+B5f0LQ93D0nS\nLiMjI4yMjMzZ+6dq8i/wSVYCN1fVu7rnQ1W1o1v+NHBMVX04yTuA7wLH0Ttt9GPgCHozlK3AicCv\ngXuAM6pqdIJt1dQnFZtYtepCRkc3TXE9Sdo3JKGqJjpLMy2TzhySXEfvq/ubkzwOrAfem2Q18CLw\nKHAuQFVtSXIDsAX4HfCJ6qXPziTnAxvpBcXVEwWDJGkw7NXMYT45c5CkqZvtmYN3SEuSGoaDJKlh\nOEiSGoaDJKlhOEiSGoaDJKlhOEiSGoaDJKlhOEiSGoaDJKlhOEiSGoaDJKlhOEiSGoaDJKlhOEiS\nGoaDJKlhOEiSGoaDJKlhOEiSGoaDJKlhOEiSGoaDJKlhOEiSGoaDJKlhOEiSGoaDJKlhOEiSGoaD\nJKlhOEiSGoaDJKlhOEiSGoaDJKlhOEjSBIaGVpJkyo+hoZUL3fVZsWShOyBJg2hs7DGgprFeZr8z\nC8CZgySpYThIkhqThkOSq5OMJXmgr7YsycYkW5PclmRp32tXJtmWZHOS1X31s5I83K1z5uzviiRp\ntuzNzOEa4APjauuA26vqSOAO4GKAJCcDh1fVEcC5wFVdfRnwOeAY4DhgfX+gSJIGy6ThUFU/AZ4d\nVz4VuLZbvrZ7vqv+7W69u4GlSZbTC5eNVfXbqnoO2AisnXn3JUlzYbrXHA6qqjGAqtoBHNTVDwae\n6Gu3vauNrz/Z1SRJA2i2L0iP/w1X6P0WbKLfdk39N2KSpHkx3fscxpIsr6qxJEPAb7r6duCQvnYr\ngKe6+vC4+p17fvtL+paHx60qSRoZGWFkZGTO3j9Vk3+BT7ISuLmq3tU9vwx4pqouS7IOOLCq1iU5\nBTivqj6YZA3wlapa012Qvhc4mt5s5V7gz7rrD+O3VVOfVGxi1aoLGR3dNMX1JGliya4TH1Nek735\nXJ1tSaiqWbsDb9KZQ5Lr6H11f3OSx4H1wD8CP0jyUeBx4DSAqrolySlJHgFeAM7p6s8m+Ty9UCjg\n0omCQZI0GPZq5jCfnDlIGgSLfebgHdKSpIbhIElqGA6SpIbhIElqGA6SpIbhIElqGA6SpIbhIElq\nGA6SpIbhIElqGA6SpIbhIElqGA6SpIbhIElqGA6SpIbhIElqGA6SpIbhIElqGA6SpIbhIElqGA6S\npIbhIElqGA6SpIbhIElqGA6SpIbhIElqGA6SpIbhIElqGA6SpIbhIElqGA6SpIbhIElqGA6SpIbh\nIElqGA6SpIbhIElqzCgckjya5OdJ7k9yT1dblmRjkq1JbkuytK/9lUm2JdmcZPVMOy9JmhsznTm8\nCAxX1Z9W1bFdbR1we1UdCdwBXAyQ5GTg8Ko6AjgXuGqG25YkzZGZhkMmeI9TgWu75Wu757vq3wao\nqruBpUmWz3D7kqQ5MNNwKOC2JD9N8vGutryqxgCqagdwUFc/GHiib90nu5okacAsmeH676mqHUn+\nCNiYZCu9wJhIJqjtoe0lfcvD3UOStMvIyAgjIyNz9v6p2tNn+RTfKFkPPA98nN51iLEkQ8CdVXVU\nkqu65e937R8CTtg1y+h7n9pzvuzJJlatupDR0U2zsCeSBEmY+mcRQJitz9UpbTWhqib6Ej4t0z6t\nlOT1SQ7olt8AnAQ8CGwAzu6anQ3c1C1vAM7s2q8BnhsfDJKkwTCT00rLgR/1vumzBPhuVW1Mci9w\nQ5KPAo8DpwFU1S1JTknyCPACcM4M+y5JmiPTDoeq+i+guVehqp4B3reHdc6f7vYkSfPHO6QlSQ3D\nQZLUMBwkSQ3DQZLUMBwkSQ3DQZLUMBwkSQ3DQZLUMBwkSQ3DQZLUMBwkSQ3DQZLUMBwkSQ3DQZLU\nMBwkaQAMDa0kyZQeQ0Mr56w/M/0b0pKkWTA29hhT/bOkY2Oz9ldBG84cJEkNw0HSvJrO6ZO5PoWi\nlqeVJM2r6Zw+6a03d6dQ1HLmIElqGA6SpIbhIElqGA6SpIbhIElqGA6SpIbhIElqGA6SpIbhIElq\nGA6SpIbhIElqGA6SpIbhIElqGA6SpIbhIElqGA6SpMa8h0OStUkeSvJwkovme/uSpMnNazgk2Q/4\nKvAB4J3AGUlWzWcfXk1GRkYWugsDw7HYzbHYzbGYO/M9czgW2FZVj1XV74DrgVPnuQ+vGh74uzkW\nuzkWuzkWc2e+w+Fg4Im+59u7miRpgCyZ5+1N9BfCm780/sY3/tWU3nTnzmd43eteO90+SZLGSVXz\n2Tx3G0vWAJdU1dru+Tqgquqyvjbz1yFJ2odU1URfwKdlvsPhNcBW4ETg18A9wBlVNTpvnZAkTWpe\nTytV1c4k5wMb6V3vuNpgkKTBM68zB0nSq8NA3SG92G6QS7IiyR1JtiR5MMmnuvqyJBuTbE1yW5Kl\nfetcmWRbks1JVi9c72dfkv2S3JdkQ/d8ZZK7unH4XpIlXX3/JNd347ApyaEL2/PZl2Rpkh8kGU3y\nyyTHLeLj4tNJfpHkgSTf7f79F8WxkeTqJGNJHuirTfk4SHJW97m6NcmZe7PtgQmHRXqD3O+BC6vq\nHcCfA+d1+7wOuL2qjgTuAC4GSHIycHhVHQGcC1y1MN2eMxcAW/qeXwZc3o3Dc8DHuvrHgGe6cfgK\n8KV57eX8uAK4paqOAt4NPMQiPC6SvAX4JHB0Vf0JvVPhZ7B4jo1r6H0m9pvScZBkGfA54BjgOGB9\nf6DsUVUNxANYA/x73/N1wEUL3a95HoN/A95H74NgeVcbAka75auAD/W1H93V7tX+AFYAPwaGgQ1d\n7Wlgv/HHB3ArcFy3/Brg6YXu/yyPxR8C/zlBfTEeF28BHgOW0QuGDcD7gd8slmMDOAx4YLrHAXA6\n8PW++tf72+3pMTAzBxb5DXJJVgKrgbvo/cOPAVTVDuCgrtn4MXqSfWeM/gn4e7r7XpK8GXi2ql7s\nXu8/Hl4ah6raCTyX5E3z29059Tbgv5Nc051m+5ckr2cRHhdV9RRwOfA4vf36LXAf8NwiPTYADtrL\n42DXuEzr+BikcNirG+T2RUkOAG4ELqiq59nzfu+TY5Tkg8BYVW1m9z6Gdn+r77WXvQX7wDj0WQIc\nDfxzVR0NvEBvJr2ojguAJAfS+y92DqM3i3gDcPIETRfLsfFK9rTv0zo+BikctgP9F49WAE8tUF/m\nTXch7UbgO1V1U1ceS7K8e32I3hQaemN0SN/q+8oYHQ/8dZJfAd8D/pLe+eKl3bUoePm+vjQO3b0z\nb6yqZ+e3y3NqO/BEVd3bPf9XemGx2I4L6J1m/VVVPdPNBH4EvAc4cJEeGzD142Ban62DFA4/Bd6e\n5LAk+9M7T7Zhgfs0H74FbKmqK/pqG4Czu+WzgZv66mfCS3ebP7drevlqVlWfqapDq+pt9P7d76iq\nvwPuBE7rmp3Fy8fhrG75NHoX5fYZ3b/pE0n+uCudCPySRXZcdB4H1iR5XZKweywW07ExfhY91ePg\nNuD93S/gltG7ZnPbpFtd6Ist4y68rKV3B/U2YN1C92ce9vd4YCewGbif3rnUtcCbgNu7sfgxcGDf\nOl8FHgF+Tu8XHAu+H7M8Jiew+4L0W4G7gYeB7wOv7ep/ANzQHSd3ASsXut9zMA7vpveFaTPwQ2Dp\nYj0ugPX0Lq4+AFwLvHaxHBvAdfS+5f8fvaA8h97F+SkdB/RCZFs3Xmfuzba9CU6S1Bik00qSpAFh\nOEiSGoaDJKlhOEiSGoaDJKlhOEiSGoaDJKlhOEiSGv8PRdFITlYTXosAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7f7f46923110>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "incorrect = filter(lambda (x,y,q,r,rv): not rv, train_5)\n",
    "print len(incorrect)\n",
    "dists = map(lambda (x,y,q,r,rv): distance(r, 5), incorrect)\n",
    "dists.sort()\n",
    "plt.hist(dists, bins=24);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3653\n",
      "2015\n"
     ]
    }
   ],
   "source": [
    "print len(train_5)\n",
    "train_5 = filter(lambda (x,y,q,r,rv): rv or (not rv and (np.abs(distance(r, 5)) < 0.1)), train_5)\n",
    "print len(train_5)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1871\n"
     ]
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAYIAAAEACAYAAAC+gnFaAAAABHNCSVQICAgIfAhkiAAAAAlwSFlz\nAAALEgAACxIB0t1+/AAAD8ZJREFUeJzt3X2spGV5x/HvTxfaWHG7vuyuAQQVVGJiV4xIW6PjC7La\nVpqmWIwWVmujtaRNk7aAabpr7R9Sa1tJa0hT1KXWILXlxZZ2FwJHa31DZRUFlqUNwoKsqQF8adpY\nufrHPAeH4xxm9szszNlzfz/JyT5zzz3PXBczO79n7uc8S6oKSVK7HjPvAiRJ82UQSFLjDAJJapxB\nIEmNMwgkqXEGgSQ1bmQQJDkmyfVJbklyc5Lf6sY3JNmdZG+SXUnWDzzmoiT7kuxJsmVg/Jwkt3eP\nOfvQtCRJOhgZdR1Bks3A5qrak+TxwBeBM4A3Ad+qqj9Jch6woarOT/Jq4Nyq+rkkLwLeV1WnJtkA\nfAE4GUi3n5Or6sFD154kaZSR3wiq6r6q2tNtfxe4FTiGfhjs7Kbt7G7T/XlpN/9zwPokm4DTgd1V\n9WBVPQDsBrZOsRdJ0goc1DmCJMcDW4DPApuq6gD0wwLY2E07Grh74GH7u7Gl4/d0Y5KkORo7CLpl\noY8Bv919M1huTSlDbteQcR5lH5KkGVk3zqQk6+iHwN9W1VXd8IEkm6rqQHce4Zvd+H7g2IGHHwPc\n2433lozfMOS5DAdJWoGqGnbAPdK43wg+ANxSVe8bGLsa2NZtbwOuGhg/GyDJqcAD3RLSLuC0JOu7\nE8endWM/oqrW7M/27dvnXoP92V+L/a3l3qomO34e+Y0gyc8CbwBuTnIT/eWcdwAXApcneTNwF3Bm\n9yF+TZLXJLkD+B793y6iqu5P8i76vzlUwDurf9JYkjRHI4Ogqv4deOwyd79ymcecu8z4h4APjVmb\nJGkGvLJ4xnq93rxLOKTs7/C2lvtby71NauQFZbOWpFZbTZK02iWhDvHJYknSGmUQSFLjDAJJapxB\nIEmNMwgkqXEGgSQ1ziCQpMYZBJLUOINAOgibNx9PkrF+Nm8+ft7lSmPxymLpICSL/3uNsWZP/K9C\nSuPyymJJ0ooZBJLUOINAkhpnEEhS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBI\nUuMMAklqnEEgSY0zCCSpcQaBJDXOIJCkxhkEktQ4g0CSGmcQSFLjDAJJapxBIEmNMwgkqXEGgSQ1\nziCQpMYZBJLUOINAkhpnEEhS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaNzIIklyS\n5ECSrwyMbU+yP8mXup+tA/ddkGRfkluTvGpgfGuS25LcnuS86bciSVqJVNWjT0heDHwXuLSqnteN\nbQe+U1V/tmTuScBHgBcCxwDXAScCAW4HXgHcC9wInFVVtw15vhpVkzQvSYBx35/B97JmJQlVlZU8\ndt2oCVX1qSTHDXveIWNnAJdV1f8BdybZB5zSzd1XVV/vCr6sm/sjQSBJmq1JzhH8ZpI9Sf4myfpu\n7Gjg7oE593RjS8f3d2OSpDkb+Y1gGe8H/qiqKskfA+8F3sLwbwnF8MBZ9jvzjh07Ht7u9Xr0er0V\nlilJa9PCwgILCwtT2dfIcwQA3dLQxxfPESx3X5LzgaqqC7v7/hXYTj8gdlTV1m78EfOW7M9zBFq1\nPEeg1WqScwTjLg2FgaP9JJsH7vsl4Kvd9tXAWUmOTPJ04ATg8/RPDp+Q5LgkRwJndXMlSXM2cmko\nyUeAHvCkJHfRP8J/WZItwEPAncBbAarqliSXA7cA3wfe3h3e/yDJucBu+uFzSVXdOv12JEkHa6yl\noVlyaUirmUtDWq1msTQkSVqjDAJJapxBIEmNMwgkqXEGgSQ1ziCQpMYZBJLUOINAkhpnEEhS4wwC\nSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuMMAklqnEEgSY0zCCSpcQaBJDXOIJCk\nxhkEktQ4g0CSGmcQSFLjDAJJapxBIEmNMwgkqXEGgSQ1ziCQpMYZBJLUOINAkhpnEEhS4wwCSWqc\nQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuMMAklqnEEgSY0zCCSpcQaBJDXOIJCkxhkE\nktS4kUGQ5JIkB5J8ZWBsQ5LdSfYm2ZVk/cB9FyXZl2RPki0D4+ckub17zNnTb0WStBLjfCP4IHD6\nkrHzgeuq6tnA9cAFAEleDTyzqk4E3gpc3I1vAP4QeCHwImD7YHhIkuZnZBBU1aeA+5cMnwHs7LZ3\ndrcXxy/tHvc5YH2STfSDZHdVPVhVDwC7ga2Tly9JmtRKzxFsrKoDAFV1H7CxGz8auHtg3v5ubOn4\nPd2YJGnO1k15fxlyu4aM040PtWPHjoe3e70evV5vCqVJ0tqxsLDAwsLCVPaVqmU/j384KTkO+HhV\nPa+7fSvQq6oDSTYDN1TVSUku7rY/2s27DXgp8LJu/tu68UfMW/JcNU5N0jwki8c2Y83G97JmJQlV\nNeyge6Rxl4bCI4/qrwa2ddvbgKsGxs/uijoVeKBbQtoFnJZkfXfi+LRuTJI0ZyOXhpJ8BOgBT0py\nF7AdeDfw90neDNwFnAlQVdckeU2SO4DvAW/qxu9P8i7gC/QPp97ZnTSWJM3ZWEtDs+TSkFYzl4a0\nWs1iaUiStEYZBJLUOINAkhpnEEhS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBI\nUuMMAklqnEEgSY0zCCSpcQaBJDXOIJCkxhkEktQ4g0CSGmcQSFLjDAJJapxBIEmNMwgkqXEGgSQ1\nziCQpMYZBJLUOINAkhpnEEhS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuMM\nAklqnEEgSY0zCCSpcQaBJDXOIJCkxhkEktQ4g0CSGmcQSFLjDAJJapxBIEmNMwgkqXETBUGSO5N8\nOclNST7fjW1IsjvJ3iS7kqwfmH9Rkn1J9iTZMmnxkqTJTfqN4CGgV1XPr6pTurHzgeuq6tnA9cAF\nAEleDTyzqk4E3gpcPOFzS5KmYNIgyJB9nAHs7LZ3drcXxy8FqKrPAeuTbJrw+SVJE5o0CArYleTG\nJG/pxjZV1QGAqroP2NiNHw3cPfDYe7oxSdIcrZvw8T9TVfcleQqwO8le+uEwTIaMDZ27Y8eOh7d7\nvR69Xm/CMiVpbVlYWGBhYWEq+0rVcp/bB7mjZDvwXeAt9M8bHEiyGbihqk5KcnG3/dFu/m3ASxe/\nPQzsp6ZVkzRtSVj+WOdHZuN7WbOShKoadsA90oqXhpI8Lsnju+2fAF4F3AxcDWzrpm0Druq2rwbO\n7uafCjywNAQkSbM3ydLQJuCKJNXt5++qaneSLwCXJ3kzcBdwJkBVXZPkNUnuAL4HvGnC2iVJUzC1\npaFpcWlIq5lLQ1qt5rI0JElaGwwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuMM\nAklqnEEgSY0zCCSpcQaBJDXOIJCkxhkEktQ4g0CSGmcQSFLjDAJJapxBIEmNMwgkqXEGgSQ1ziCQ\npMYZBJLUOINAkhpnEEhS4wwCSWqcQSBJjTMIJKlxBoEkNc4gkKTGGQSS1DiDQJIaZxBIUuMMAklq\nnEEgSY0zCCSpcQaBJDXOIJCkxhkEktQ4g0CSGmcQSFLjDAJJapxBIEmNMwgkqXEGgSQ1ziCQpMbN\nPAiSbE1yW5Lbk5w36+eXJD3STIMgyWOAvwROB54LvD7Jc2ZZw7wtLCzMu4RDaq33t9at5ddvLfc2\nqVl/IzgF2FdVX6+q7wOXAWfMuIa5WutvxrXe31q3ll+/tdzbpGYdBEcDdw/c3t+NSZLmZN2Mny9D\nxmrGNTzCe97zHj75yU+ONXfdunVcccUVh7giSZqtVM3uczjJqcCOqtra3T4fqKq6cGDOXINBkg5X\nVTXsYHukWQfBY4G9wCuAbwCfB15fVbfOrAhJ0iPMdGmoqn6Q5FxgN/3zE5cYApI0XzP9RiBJWn3m\nfmVxkg1JdifZm2RXkvXLzDu2u/+WJF9N8rRZ17oS4/bXzT0qyf4kF82yxkmM01+Sn0ry6SQ3J9mT\n5HXzqPVgjLrwMcmRSS5Lsi/JZw6X9yOM1dvvJPla91pdm+TYedS5UuNetJrkl5M8lOTkWdY3qXH6\nS/K67jW8OcmHR+60qub6A1wI/H63fR7w7mXm3QC8vNt+HPDj8659mv119/8F8GHgonnXPc3+gBOA\nZ3bbTwXuBZ4w79ofpafHAHcAxwFHAHuA5yyZ8xvA+7vtXwEum3fdU+ztpYt/v4C3HS69jdtfN+/x\nwCeATwMnz7vuKb9+JwBfXPw7Bjx51H7n/o2A/gVlO7vtncAvLp2Q5CTgsVV1PUBV/XdV/c/sSpzI\nyP4AkrwA2Ej//MnhZGR/VXVHVf1Ht/0N4JvAU2ZW4cEb58LHwb4/Rv8XIA4HI3urqk8M/P36LIfX\ntT7jXrT6LvoHMf87y+KmYJz+fh34q6r6NkBV/deona6GINhYVQcAquo+hn9APAt4MMk/JPlikguT\nrOjXpOZgZH9dL38K/B7Dr7VYzcZ5/R6W5BTgiMVgWKXGufDx4TlV9QPggSRPnE15EznYizp/DfiX\nQ1rRdI3sL8kW4JiqumaWhU3JOK/fs4BnJ/lUtyR7+qidzuS3hpJcC2waHKJ/IdkfjLmLdcCLgS30\n/yNcDmwDPji9KlduCv29Hfjnqrqny7dVFQZT6G9xP08FLgV+dXrVHRLjXPi4dE6GzFmNxr6oM8kb\ngRfQXyo6XDxqf91B158D54x4zGo1zuu3jv7y0EuApwH/luS5i98QhplJEFTVacvdl+RAkk1VdSDJ\nZvrLBkvtB26qqq93j7kSeBGrJAim0N9PAy9O8nbgKOCIJN+pqnccopIPyhT6I8lRwD8B76iqGw9R\nqdOyn/5foEXH0D+vMehu4Fjg3u76mCdU1f0zqm8S4/RGklcCFwAv6ZYgDhej+juK/j94udCFwmbg\nqiSvraovza7MFRvn9dsPfKaqHgLuTLIXOJH+eYOhVsPS0NX0j+6hn9JXDZlzI7AhyZO62y8Hbjn0\npU3FyP6q6o1VdXxVPQP4XeDS1RICYxjZX5IjgCuBnVX1j7MrbcVuBE5IclySI4Gz6Pc56OP88Kjy\nTOD6GdY3iZG9JXk+cDHw2qr61hxqnMSj9ldV366qjVX1jKp6Ov1zIL9wmIQAjPfevJL+ZyRJnkw/\nBP7zUfe6Cs6CPxG4jv4Vx9cCP9mNvwD464F5rwC+3P18AFg379qn2d/A/HM4vH5raGR/wBvon5T7\nEnBT9+fz5l37iL62dj3tA87vxt4J/Hy3/WP0lyj30f8wOX7eNU+xt2vpX/m/+HpdOe+ap9nfkrnX\ncxj91tC4/QHvBb7WfV6eOWqfXlAmSY1bDUtDkqQ5MggkqXEGgSQ1ziCQpMYZBJLUOINAkhpnEEhS\n4wwCSWrc/wNQELpK2pMM9wAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<matplotlib.figure.Figure at 0x7fe67cfa46d0>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "incorrect = filter(lambda (x,y,q,r,rv): not rv, train_7)\n",
    "print len(incorrect)\n",
    "dists = map(lambda (x,y,q,r,rv): distance(r, 7), incorrect)\n",
    "dists.sort()\n",
    "plt.hist(dists, bins=24);"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2076\n",
      "2076\n"
     ]
    }
   ],
   "source": [
    "print len(train_7)\n",
    "train_7 = filter(lambda (x,y,q,r,rv): rv or (not rv and (np.abs(distance(r, 7)) < 0.1)), train_7)\n",
    "print len(train_7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# filter according to EXIF data"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = train_1\n",
    "train_with_exif = []\n",
    "for x,y,q,r,rv in progress_iter(train, True):\n",
    "    tags = exifread.process_file(open(x, 'rb'))\n",
    "    train_with_exif.append((x,y,q,r,rv,tags))\n",
    "train_1 = train_with_exif"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print len(train_1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print list(set([str(t['Image Make']) for x,y,q,r,rv,t in train_1]))\n",
    "print list(set([str(t['Image Model']) for x,y,q,r,rv,t in train_1]))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "c = 1\n",
    "c1 = lambda t: 'EXIF ExifImageWidth' in t\n",
    "c2 = lambda t: 'EXIF ExifImageLength' in t\n",
    "c3 = lambda t: distance((int(str(t['EXIF ExifImageWidth'])), int(str(t['EXIF ExifImageLength']))), c) < 0.1\n",
    "cond = [\n",
    "    lambda t: 'Image Make' in t and str(t['Image Make']) in ['Huawei', 'LGE'],\n",
    "    lambda t: 'Image Model' in t and str(t['Image Model']) in ['Nexus 5X', 'Nexus 6P'],\n",
    "#     lambda t: c1(t) and c2(t) and c3(t),\n",
    "#     lambda t: 'Image Orientation' in t and str(t['Image Orientation']) == 'Horizontal (normal)'\n",
    "]\n",
    "all_cond = lambda (x,y,q,r,rv,t): all(map(lambda c: c(t), cond))\n",
    "print len(filter(all_cond, train_1))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print len(train_1)\n",
    "train_1 = filter(all_cond, train_1)\n",
    "print len(train_1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = train_0\n",
    "train_with_exif = []\n",
    "for x,y,q,r,rv in progress_iter(train, True):\n",
    "    tags = exifread.process_file(open(x, 'rb'))\n",
    "    train_with_exif.append((x,y,q,r,rv,tags))\n",
    "train_0 = train_with_exif"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "c = 0\n",
    "c1 = lambda t: 'EXIF ExifImageWidth' in t\n",
    "c2 = lambda t: 'EXIF ExifImageLength' in t\n",
    "c3 = lambda t: np.abs(distance((int(str(t['EXIF ExifImageWidth'])), int(str(t['EXIF ExifImageLength']))), c)) < 0.01\n",
    "cond = [\n",
    "    lambda t: str(t['Image Make']) == 'HTC',\n",
    "    lambda t: str(t['Image Model']) == 'HTC One',\n",
    "    lambda t: c1(t) and c2(t) and c3(t),\n",
    "    lambda t: 'Image Orientation' in t and str(t['Image Orientation']) == 'Horizontal (normal)'\n",
    "]\n",
    "all_cond = lambda (x,y,q,r,rv,t): all(map(lambda c: c(t), cond))\n",
    "print len(filter(all_cond, train_0))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "print len(train_0)\n",
    "train_0 = filter(all_cond, train_0)\n",
    "print len(train_0)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = train_9\n",
    "train_with_exif = []\n",
    "for x,y,q,r,rv in progress_iter(train, True):\n",
    "    tags = exifread.process_file(open(x, 'rb'))\n",
    "    train_with_exif.append((x,y,q,r,rv,tags))\n",
    "train_9 = train_with_exif"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "c = 9\n",
    "\n",
    "def f(v):\n",
    "    return float(str(v).replace(', 0]', '').replace('[', ''))\n",
    "\n",
    "c1 = lambda t: 'EXIF ExifImageWidth' in t\n",
    "c2 = lambda t: 'EXIF ExifImageLength' in t\n",
    "c3 = lambda t: np.abs(distance((f(t['EXIF ExifImageWidth']), f(t['EXIF ExifImageLength'])), c)) < 0.1\n",
    "cond = [\n",
    "    lambda t: 'Image Make' in t and str(t['Image Make']) == 'Apple',\n",
    "    lambda t: 'Image Model' in t and str(t['Image Model']) == 'iPhone 6',\n",
    "    lambda t: 'EXIF LensMake' in t and str(t['EXIF LensMake']) == 'Apple',\n",
    "    lambda t: 'EXIF LensModel' in t and str(t['EXIF LensModel']) == 'iPhone 6 back camera 4.15mm f/2.2',\n",
    "    lambda t: c1(t) and c2(t) and c3(t),\n",
    "#     lambda t: 'Image Orientation' in t and str(t['Image Orientation']) == 'Horizontal (normal)'\n",
    "]\n",
    "all_cond = lambda (x,y,q,r,rv,t): all(map(lambda c: c(t), cond))\n",
    "print len(filter(all_cond, train_9))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "93d79bcb14364723a36d5ff3475422f3",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "train = train_2\n",
    "train_with_exif = []\n",
    "for x,y,q,r,rv in progress_iter(train, True):\n",
    "    tags = exifread.process_file(open(x, 'rb'))\n",
    "    train_with_exif.append((x,y,q,r,rv,tags))\n",
    "train_2 = train_with_exif"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5355\n"
     ]
    }
   ],
   "source": [
    "c = 2\n",
    "c1 = lambda t: 'EXIF ExifImageWidth' in t\n",
    "c2 = lambda t: 'EXIF ExifImageLength' in t\n",
    "c3 = lambda t: np.abs(distance((float(str(t['EXIF ExifImageWidth'])), float(str(t['EXIF ExifImageLength']))), c)) < 0.01\n",
    "cond = [\n",
    "    lambda t: 'Image Make' in t and str(t['Image Make']) == 'Motorola',\n",
    "    lambda t: 'Image Model' in t and str(t['Image Model']) == 'XT1080',\n",
    "    lambda t: c1(t) and c2(t) and c3(t),\n",
    "#     lambda t: 'Image Orientation' in t and str(t['Image Orientation']) == 'Horizontal (normal)'\n",
    "]\n",
    "all_cond = lambda (x,y,q,r,rv,t): all(map(lambda c: c(t), cond))\n",
    "print len(filter(all_cond, train_2))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5355\n",
      "5355\n"
     ]
    }
   ],
   "source": [
    "print len(train_2)\n",
    "train_2 = filter(all_cond, train_2)\n",
    "print len(train_2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ac658b6f832048f5bc6f1c6a92b9af45",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "train = train_6\n",
    "train_with_exif = []\n",
    "for x,y,q,r,rv in progress_iter(train, True):\n",
    "    tags = exifread.process_file(open(x, 'rb'))\n",
    "    train_with_exif.append((x,y,q,r,rv,tags))\n",
    "train_6 = train_with_exif"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def f(v):\n",
    "    return float(str(v).replace(', 0]', '').replace('[', ''))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2912\n"
     ]
    }
   ],
   "source": [
    "c = 6\n",
    "c1 = lambda t: 'EXIF ExifImageWidth' in t\n",
    "c2 = lambda t: 'EXIF ExifImageLength' in t\n",
    "c3 = lambda t: np.abs(distance((f(t['EXIF ExifImageWidth']), f(t['EXIF ExifImageLength'])), c)) < 0.1\n",
    "cond = [\n",
    "    lambda t: 'Image Make' in t and str(t['Image Make']).lower() == 'samsung',\n",
    "    lambda t: 'Image Model' in t and str(t['Image Model']) in ['GT-I9505', 'SCH-I545', 'GT-I9500'],\n",
    "    lambda t: c1(t) and c2(t) and c3(t),\n",
    "#     lambda t: 'Image Orientation' in t and str(t['Image Orientation']) == 'Horizontal (normal)'\n",
    "]\n",
    "all_cond = lambda (x,y,q,r,rv,t): all(map(lambda c: c(t), cond))\n",
    "print len(filter(all_cond, train_6))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bf1f6eb634334091967a794cbe7c9b82",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "No handlers could be found for logger \"exifread\"\n"
     ]
    }
   ],
   "source": [
    "train = train_8\n",
    "train_with_exif = []\n",
    "for x,y,q,r,rv in progress_iter(train, True):\n",
    "    try:\n",
    "        tags = exifread.process_file(open(x, 'rb'))\n",
    "        train_with_exif.append((x,y,q,r,rv,tags))\n",
    "    except:\n",
    "        pass\n",
    "train_8 = train_with_exif"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4596\n"
     ]
    }
   ],
   "source": [
    "c = 8\n",
    "\n",
    "def f(v):\n",
    "    return float(str(v).replace(', 0]', '').replace('[', ''))\n",
    "\n",
    "c1 = lambda t: 'EXIF ExifImageWidth' in t\n",
    "c2 = lambda t: 'EXIF ExifImageLength' in t\n",
    "c3 = lambda t: np.abs(distance((f(t['EXIF ExifImageWidth']), f(t['EXIF ExifImageLength'])), c)) < 0.01\n",
    "cond = [\n",
    "    lambda t: 'Image Make' in t and str(t['Image Make']) == 'Apple',\n",
    "    lambda t: 'Image Model' in t and str(t['Image Model']) == 'iPhone 4S',\n",
    "    lambda t: 'EXIF LensMake' in t and str(t['EXIF LensMake']) == 'Apple',\n",
    "    lambda t: 'EXIF LensModel' in t and str(t['EXIF LensModel']) == 'iPhone 4S back camera 4.28mm f/2.4',\n",
    "    lambda t: c1(t) and c2(t) and c3(t),\n",
    "#     lambda t: 'Image Orientation' in t and str(t['Image Orientation']) == 'Horizontal (normal)'\n",
    "]\n",
    "all_cond = lambda (x,y,q,r,rv,t): all(map(lambda c: c(t), cond))\n",
    "print len(filter(all_cond, train_8))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "5807\n",
      "4596\n"
     ]
    }
   ],
   "source": [
    "print len(train_8)\n",
    "train_8 = filter(all_cond, train_8)\n",
    "print len(train_8)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "41684bded76743228177b4e91b6a985a",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "train = train_3\n",
    "train_with_exif = []\n",
    "for x,y,q,r,rv in progress_iter(train, True):\n",
    "    try:\n",
    "        tags = exifread.process_file(open(x, 'rb'))\n",
    "        train_with_exif.append((x,y,q,r,rv,tags))\n",
    "    except:\n",
    "        pass\n",
    "train_3 = train_with_exif"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1804\n"
     ]
    }
   ],
   "source": [
    "c = 3\n",
    "c1 = lambda t: 'EXIF ExifImageWidth' in t\n",
    "c2 = lambda t: 'EXIF ExifImageLength' in t\n",
    "c3 = lambda t: distance((float(str(t['EXIF ExifImageWidth'])), float(str(t['EXIF ExifImageLength']))), c) < 0.1\n",
    "cond = [\n",
    "    lambda t: 'Image Make' in t and str(t['Image Make']) == 'motorola',\n",
    "    lambda t: 'Image Model' in t and str(t['Image Model']) == 'Nexus 6',\n",
    "    lambda t: c1(t) and c2(t) and c3(t),\n",
    "#     lambda t: 'Image Orientation' in t and str(t['Image Orientation']) == 'Horizontal (normal)'\n",
    "]\n",
    "all_cond = lambda (x,y,q,r,rv,t): all(map(lambda c: c(t), cond))\n",
    "print len(filter(all_cond, train_3))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1893\n",
      "1804\n"
     ]
    }
   ],
   "source": [
    "print len(train_3)\n",
    "train_3 = filter(all_cond, train_3)\n",
    "print len(train_3)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d7a8e333b9854ec8ba8b204eff7f936d",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "train = train_5\n",
    "train_with_exif = []\n",
    "for x,y,q,r,rv in progress_iter(train, True):\n",
    "    try:\n",
    "        tags = exifread.process_file(open(x, 'rb'))\n",
    "        train_with_exif.append((x,y,q,r,rv,tags))\n",
    "    except:\n",
    "        pass\n",
    "train_5 = train_with_exif"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2015\n"
     ]
    }
   ],
   "source": [
    "c = 5\n",
    "\n",
    "def f(v):\n",
    "    return float(str(v).replace(', 0]', '').replace('[', ''))\n",
    "\n",
    "c1 = lambda t: 'EXIF ExifImageWidth' in t\n",
    "c2 = lambda t: 'EXIF ExifImageLength' in t\n",
    "c3 = lambda t: distance((f(t['EXIF ExifImageWidth']), f(t['EXIF ExifImageLength'])), c) < 0.1\n",
    "cond = [\n",
    "    lambda t: 'Image Make' in t and str(t['Image Make']).lower() == 'samsung',\n",
    "    lambda t: 'Image Model' in t and str(t['Image Model']) in ['SAMSUNG-SM-N900A', 'SM-N9005'],\n",
    "    lambda t: c1(t) and c2(t) and c3(t),\n",
    "#     lambda t: 'Image Orientation' in t and str(t['Image Orientation']) == 'Horizontal (normal)'\n",
    "]\n",
    "all_cond = lambda (x,y,q,r,rv,t): all(map(lambda c: c(t), cond))\n",
    "print len(filter(all_cond, train_5))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ead51390a10b4fb990aed1f19817787e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "train = train_7\n",
    "train_with_exif = []\n",
    "for x,y,q,r,rv in progress_iter(train, True):\n",
    "    try:\n",
    "        tags = exifread.process_file(open(x, 'rb'))\n",
    "        train_with_exif.append((x,y,q,r,rv,tags))\n",
    "    except:\n",
    "        pass\n",
    "train_7 = train_with_exif"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1328\n"
     ]
    }
   ],
   "source": [
    "c = 7\n",
    "\n",
    "def f(v):\n",
    "    return float(str(v).replace(', 0]', '').replace('[', ''))\n",
    "\n",
    "c1 = lambda t: 'EXIF ExifImageWidth' in t\n",
    "c2 = lambda t: 'EXIF ExifImageLength' in t\n",
    "c3 = lambda t: np.abs(distance((f(t['EXIF ExifImageWidth']), f(t['EXIF ExifImageLength'])), c)) < 0.1\n",
    "cond = [\n",
    "    lambda t: 'Image Make' in t and str(t['Image Make']) == 'SONY',\n",
    "    lambda t: 'Image Model' in t and str(t['Image Model']) == 'NEX-7',\n",
    "    lambda t: c1(t) and c2(t) and c3(t),\n",
    "#     lambda t: 'Image Orientation' in t and str(t['Image Orientation']) == 'Horizontal (normal)'\n",
    "]\n",
    "all_cond = lambda (x,y,q,r,rv,t): all(map(lambda c: c(t), cond))\n",
    "print len(filter(all_cond, train_7))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2076\n",
      "1328\n"
     ]
    }
   ],
   "source": [
    "print len(train_7)\n",
    "train_7 = filter(all_cond, train_7)\n",
    "print len(train_7)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# crop and save blocks to disk"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "BLOCK_SIZE = 48\n",
    "crop_size = 1024"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "c = 1\n",
    "class_train = train_1[:N_IMAGES_NEEDED[c]]\n",
    "N_BLOCKS = int(np.round(len(class_train)/float(BLOCK_SIZE)))\n",
    "print '{0} blocks ...'.format(N_BLOCKS)\n",
    "\n",
    "RNG(seed=9999 + c).shuffle(class_train)\n",
    "\n",
    "class_blocks = []\n",
    "for _ in xrange(N_BLOCKS):\n",
    "    class_blocks.append([])\n",
    "pos = 0\n",
    "N_IMAGES_PER_ANDREAS_BLOCK = []\n",
    "for x,_,_,_,_,_ in progress_iter(class_train, True):\n",
    "    x = Image.open(x)\n",
    "    w = x.size[0]\n",
    "    h = x.size[1]\n",
    "    x = x.crop((w/2-crop_size/2, h/2-crop_size/2,\n",
    "                w/2+crop_size/2, h/2+crop_size/2))\n",
    "    class_blocks[pos % N_BLOCKS].append(np.asarray(x, dtype=np.uint8))\n",
    "    pos += 1\n",
    "for b in xrange(N_BLOCKS):\n",
    "    X_block = np.asarray(class_blocks[b], dtype=np.uint8)\n",
    "    N_IMAGES_PER_ANDREAS_BLOCK.append(len(X_block))\n",
    "    np.save('../data/X_andreas_{0}_{1}.npy'.format(c, b), X_block)\n",
    "print N_IMAGES_PER_ANDREAS_BLOCK"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### class 0"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "c = 0\n",
    "class_train = train_0\n",
    "\n",
    "RNG(seed=9999 + c).shuffle(class_train)\n",
    "class_train = class_train[:N_IMAGES_NEEDED[c]]\n",
    "print len(class_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def make_blocks(class_train, c, b_start=0):\n",
    "    N_BLOCKS = int(np.round(len(class_train)/float(BLOCK_SIZE)))\n",
    "    print '{0} blocks ...'.format(N_BLOCKS)\n",
    "    \n",
    "    class_blocks = []\n",
    "    for _ in xrange(N_BLOCKS):\n",
    "        class_blocks.append([])\n",
    "    pos = 0\n",
    "    N_IMAGES_PER_ANDREAS_BLOCK = []\n",
    "    for x,_,_,_,_,_ in progress_iter(class_train, True):\n",
    "        x = Image.open(x)\n",
    "        w = x.size[0]\n",
    "        h = x.size[1]\n",
    "        x = x.crop((w/2-crop_size/2, h/2-crop_size/2,\n",
    "                    w/2+crop_size/2, h/2+crop_size/2))\n",
    "        class_blocks[pos % N_BLOCKS].append(np.asarray(x, dtype=np.uint8))\n",
    "        pos += 1\n",
    "    for b in xrange(N_BLOCKS):\n",
    "        X_block = np.asarray(class_blocks[b], dtype=np.uint8)\n",
    "        N_IMAGES_PER_ANDREAS_BLOCK.append(len(X_block))\n",
    "        np.save('../data/X_andreas_{0}_{1}.npy'.format(c, b + b_start), X_block)\n",
    "    print N_IMAGES_PER_ANDREAS_BLOCK"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "make_blocks(class_train[:800], 0, 0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "make_blocks(class_train[800:1600], 0, 17)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "make_blocks(class_train[1600:2400], 0, 34)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "make_blocks(class_train[2400:3200], 0, 51)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "make_blocks(class_train[3200:], 0, 68)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 9"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3734\n"
     ]
    }
   ],
   "source": [
    "c = 9\n",
    "class_train = train_9\n",
    "class_train.sort(key=lambda (x,y,q,r,rv,t): -q)\n",
    "class_train = class_train[:N_IMAGES_NEEDED[c]]\n",
    "\n",
    "RNG(seed=9999 + c).shuffle(class_train)\n",
    "print len(class_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 124,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20 blocks ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b37ec155207e44b0ac53bdcf451263c2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48]\n",
      "20 blocks ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d9b3a099e75f4b88877f76e8d8ee903f",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48]\n",
      "20 blocks ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e413e6ccaf344c56be109b19e945d3e0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48]\n",
      "18 blocks ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "e755f14adb9c48429e0ed764b3399de9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[48, 48, 48, 48, 48, 48, 48, 48, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47]\n"
     ]
    }
   ],
   "source": [
    "make_blocks(class_train[:960], c, 0)\n",
    "make_blocks(class_train[960:2*960], c, 20)\n",
    "make_blocks(class_train[2*960:3*960], c, 40)\n",
    "make_blocks(class_train[3*960:], c, 60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 2"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "4026\n"
     ]
    }
   ],
   "source": [
    "c = 2\n",
    "class_train = train_2\n",
    "class_train.sort(key=lambda (x,y,q,r,rv,t): -q)\n",
    "class_train = class_train[:N_IMAGES_NEEDED[c]]\n",
    "\n",
    "RNG(seed=9999 + c).shuffle(class_train)\n",
    "print len(class_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "make_blocks(class_train[:960], c, 0)\n",
    "make_blocks(class_train[960:2*960], c, 20)\n",
    "make_blocks(class_train[2*960:3*960], c, 40)\n",
    "make_blocks(class_train[3*960:], c, 60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2915\n"
     ]
    }
   ],
   "source": [
    "c = 6\n",
    "class_train = train_6\n",
    "class_train.sort(key=lambda (x,y,q,r,rv,t): -q)\n",
    "class_train = class_train[:N_IMAGES_NEEDED[c]]\n",
    "\n",
    "RNG(seed=9999 + c).shuffle(class_train)\n",
    "print len(class_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "make_blocks(class_train[:960], c, 0)\n",
    "make_blocks(class_train[960:2*960], c, 20)\n",
    "make_blocks(class_train[2*960:3*960], c, 40)\n",
    "make_blocks(class_train[3*960:], c, 60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "3931\n"
     ]
    }
   ],
   "source": [
    "c = 8\n",
    "class_train = train_8\n",
    "class_train.sort(key=lambda (x,y,q,r,rv,t): -q)\n",
    "class_train = class_train[:N_IMAGES_NEEDED[c]]\n",
    "\n",
    "RNG(seed=9999 + c).shuffle(class_train)\n",
    "print len(class_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20 blocks ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "31141b3930084952baaceb821d5544c0",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48]\n",
      "20 blocks ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "1ce8bcb174ea4bb7bcfb40f64eb17a63",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48]\n",
      "20 blocks ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "d8ab8ad451ac41c09b41f9b3772f4cbe",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48]\n",
      "22 blocks ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "a4da01db0245416fa58a1a5234e8e280",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 47, 47, 47, 47, 47]\n"
     ]
    }
   ],
   "source": [
    "make_blocks(class_train[:960], c, 0)\n",
    "make_blocks(class_train[960:2*960], c, 20)\n",
    "make_blocks(class_train[2*960:3*960], c, 40)\n",
    "make_blocks(class_train[3*960:], c, 60)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 3"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1804\n"
     ]
    }
   ],
   "source": [
    "c = 3\n",
    "class_train = train_3\n",
    "class_train.sort(key=lambda (x,y,q,r,rv,t): -q)\n",
    "class_train = class_train[:N_IMAGES_NEEDED[c]]\n",
    "\n",
    "RNG(seed=9999 + c).shuffle(class_train)\n",
    "print len(class_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20 blocks ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "ef5e2e531004481db4e21dd5f29b792e",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48]\n",
      "18 blocks ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "bf5d185ac92541ad99e365e8f77dbdc9",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 47, 46, 46]\n"
     ]
    }
   ],
   "source": [
    "make_blocks(class_train[:960], c, 0)\n",
    "make_blocks(class_train[960:], c, 20)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### class 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "2015\n"
     ]
    }
   ],
   "source": [
    "c = 5\n",
    "class_train = train_5\n",
    "class_train.sort(key=lambda (x,y,q,r,rv,t): -q)\n",
    "class_train = class_train[:N_IMAGES_NEEDED[c]]\n",
    "\n",
    "RNG(seed=9999 + c).shuffle(class_train)\n",
    "print len(class_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20 blocks ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "c3284b5d031741228e68c7bf1c6211ea",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48]\n",
      "20 blocks ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "594888f812464d04a3204799238b0b64",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48]\n",
      "2 blocks ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "af1bffc9006740e0bddbc933e86dccc2",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[48, 47]\n"
     ]
    }
   ],
   "source": [
    "make_blocks(class_train[:960], c, 0)\n",
    "make_blocks(class_train[960:2*960], c, 20)\n",
    "make_blocks(class_train[2*960:], c, 40)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "### class 7"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1328\n"
     ]
    }
   ],
   "source": [
    "c = 7\n",
    "class_train = train_7\n",
    "class_train.sort(key=lambda (x,y,q,r,rv,t): -q)\n",
    "class_train = class_train[:N_IMAGES_NEEDED[c]]\n",
    "\n",
    "RNG(seed=9999 + c).shuffle(class_train)\n",
    "print len(class_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "20 blocks ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "b226c100f20d44c19557e1bb85b2b3aa",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48, 48]\n",
      "8 blocks ...\n"
     ]
    },
    {
     "data": {
      "application/vnd.jupyter.widget-view+json": {
       "model_id": "09065a5c4c574a638e814958b37a503c",
       "version_major": 2,
       "version_minor": 0
      },
      "text/plain": [
       "A Jupyter Widget"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "\n",
      "[46, 46, 46, 46, 46, 46, 46, 46]\n"
     ]
    }
   ],
   "source": [
    "make_blocks(class_train[:960], c, 0)\n",
    "make_blocks(class_train[960:], c, 20)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 2",
   "language": "python",
   "name": "python2"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 2
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython2",
   "version": "2.7.12"
  },
  "latex_envs": {
   "LaTeX_envs_menu_present": true,
   "autocomplete": true,
   "bibliofile": "biblio.bib",
   "cite_by": "apalike",
   "current_citInitial": 1,
   "eqLabelWithNumbers": true,
   "eqNumInitial": 1,
   "hotkeys": {
    "equation": "Ctrl-E",
    "itemize": "Ctrl-I"
   },
   "labels_anchors": false,
   "latex_user_defs": false,
   "report_style_numbering": false,
   "user_envs_cfg": false
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
